//===--- GenValueWitness.cpp - IR generation for value witnesses ----------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
//  This file implements IR generation for value witnesses in Polarboy.
//
//  Value witnesses are (predominantly) functions that implement the basic
//  operations for copying and destroying values.
//
//  In the comments throughout this file, three type names are used:
//    'B' is the type of a fixed-size buffer
//    'T' is the type which implements a protocol
//
//===----------------------------------------------------------------------===//

#include "polarphp/ast/AstContext.h"
#include "polarphp/ast/Types.h"
#include "polarphp/irgen/Linking.h"
#include "polarphp/pil/lang/TypeLowering.h"

#include "llvm/ADT/SmallString.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"

#include "polarphp/irgen/internal/ConstantBuilder.h"
#include "polarphp/irgen/internal/Explosion.h"
#include "polarphp/irgen/internal/FixedTypeInfo.h"
#include "polarphp/irgen/internal/GenEnum.h"
#include "polarphp/irgen/internal/GenMeta.h"
#include "polarphp/irgen/internal/GenOpaque.h"
#include "polarphp/irgen/internal/IRGenDebugInfo.h"
#include "polarphp/irgen/internal/IRGenFunction.h"
#include "polarphp/irgen/internal/IRGenModule.h"
#include "polarphp/irgen/internal/StructLayout.h"
#include "polarphp/irgen/internal/TypeInfo.h"
#include "polarphp/irgen/internal/GenValueWitness.h"

using namespace polar;
using namespace irgen;

const char *irgen::getValueWitnessName(ValueWitness witness) {
   switch (witness) {
#define CASE(NAME) case ValueWitness::NAME: return #NAME;
      CASE(AssignWithCopy)
      CASE(AssignWithTake)
      CASE(Destroy)
      CASE(InitializeBufferWithCopyOfBuffer)
      CASE(InitializeWithCopy)
      CASE(InitializeWithTake)
      CASE(GetEnumTag)
      CASE(DestructiveProjectEnumData)
      CASE(DestructiveInjectEnumTag)
      CASE(Size)
      CASE(Flags)
      CASE(ExtraInhabitantCount)
      CASE(Stride)
      CASE(GetEnumTagSinglePayload)
      CASE(StoreEnumTagSinglePayload)
#undef CASE
   }
   llvm_unreachable("bad value witness kind");
}

namespace {
/// An operation to be performed for various kinds of packing.
struct DynamicPackingOperation {
   virtual ~DynamicPackingOperation() = default;

   /// Emit the operation at a concrete packing kind.
   ///
   /// Immediately after this call, there will be an unconditional
   /// branch to the continuation block.
   virtual void emitForPacking(IRGenFunction &IGF,
                               PILType T,
                               const TypeInfo &type,
                               FixedPacking packing) = 0;

   /// Given that we are currently at the beginning of the
   /// continuation block, complete the operation.
   virtual void complete(IRGenFunction &IGF) = 0;
};

/// A class for merging a particular kind of value across control flow.
template <class T> class DynamicPackingPHIMapping;

/// An implementation of DynamicPackingPHIMapping for a single LLVM value.
template <> class DynamicPackingPHIMapping<llvm::Value*> {
   llvm::PHINode *PHI = nullptr;
public:
   void collect(IRGenFunction &IGF, llvm::Value *value) {
      // Add the result to the phi, creating it (unparented) if necessary.
      if (!PHI) PHI = llvm::PHINode::Create(value->getType(), 2,
                                            "dynamic-packing.result");
      PHI->addIncoming(value, IGF.Builder.GetInsertBlock());
   }
   void complete(IRGenFunction &IGF) {
      assert(PHI);
      IGF.Builder.Insert(PHI);
   }
   llvm::Value *get(IRGenFunction &IGF, PILType T, const TypeInfo &type) {
      assert(PHI);
      return PHI;
   }
};

/// An implementation of DynamicPackingPHIMapping for Addresses.
template <> class DynamicPackingPHIMapping<Address>
   : private DynamicPackingPHIMapping<llvm::Value*> {
   using super = DynamicPackingPHIMapping<llvm::Value *>;

public:
   void collect(IRGenFunction &IGF, Address value) {
      super::collect(IGF, value.getAddress());
   }
   void complete(IRGenFunction &IGF) {
      super::complete(IGF);
   }
   Address get(IRGenFunction &IGF, PILType T, const TypeInfo &type) {
      return type.getAddressForPointer(super::get(IGF, T, type));
   }
};

/// An implementation of packing operations based around a lambda.
template <class ResultTy, class FnTy>
class LambdaDynamicPackingOperation : public DynamicPackingOperation {
   FnTy Fn;
   DynamicPackingPHIMapping<ResultTy> Mapping;
public:
   explicit LambdaDynamicPackingOperation(FnTy &&fn) : Fn(fn) {}
   void emitForPacking(IRGenFunction &IGF, PILType T, const TypeInfo &type,
                       FixedPacking packing) override {
      Mapping.collect(IGF, Fn(IGF, T, type, packing));
   }

   void complete(IRGenFunction &IGF) override {
      Mapping.complete(IGF);
   }

   ResultTy get(IRGenFunction &IGF, PILType T, const TypeInfo &type) {
      return Mapping.get(IGF, T, type);
   }
};

/// A partial specialization for lambda-based packing operations
/// that return 'void'.
template <class FnTy>
class LambdaDynamicPackingOperation<void, FnTy>
   : public DynamicPackingOperation {
   FnTy Fn;
public:
   explicit LambdaDynamicPackingOperation(FnTy &&fn) : Fn(fn) {}
   void emitForPacking(IRGenFunction &IGF, PILType T, const TypeInfo &type,
                       FixedPacking packing) override {
      Fn(IGF, T, type, packing);
   }
   void complete(IRGenFunction &IGF) override {}
   void get(IRGenFunction &IGF, PILType T, const TypeInfo &type) {}
};
} // end anonymous namespace

/// Dynamic check for the enabling conditions of different kinds of
/// packing into a fixed-size buffer, and perform an operation at each
/// of them.
static void emitDynamicPackingOperation(IRGenFunction &IGF,
                                        PILType T,
                                        const TypeInfo &type,
                                        DynamicPackingOperation &operation) {
   auto indirectBB = IGF.createBasicBlock("dynamic-packing.indirect");
   auto directBB = IGF.createBasicBlock("dynamic-packing.direct");
   auto contBB = IGF.createBasicBlock("dynamic-packing.cont");

   // Branch.
   auto isInline = type.isDynamicallyPackedInline(IGF, T);
   IGF.Builder.CreateCondBr(isInline, directBB, indirectBB);

   // Emit the indirect path.
   IGF.Builder.emitBlock(indirectBB); {
      ConditionalDominanceScope condition(IGF);
      operation.emitForPacking(IGF, T, type, FixedPacking::Allocate);
      IGF.Builder.CreateBr(contBB);
   }

   // Emit the direct path.
   IGF.Builder.emitBlock(directBB); {
      ConditionalDominanceScope condition(IGF);
      operation.emitForPacking(IGF, T, type, FixedPacking::OffsetZero);
      IGF.Builder.CreateBr(contBB);
   }

   // Enter the continuation block and add the PHI if required.
   IGF.Builder.emitBlock(contBB);
   operation.complete(IGF);
}

/// A helper function for creating a lambda-based DynamicPackingOperation.
template <class ResultTy, class FnTy>
LambdaDynamicPackingOperation<ResultTy, FnTy>
makeLambdaDynamicPackingOperation(FnTy &&fn) {
   return LambdaDynamicPackingOperation<ResultTy, FnTy>(std::move(fn));
}

/// Perform an operation on a type that requires dynamic packing.
template <class ResultTy, class... ArgTys, class... ParamTys>
static ResultTy emitForDynamicPacking(IRGenFunction &IGF,
                                      ResultTy (*fn)(ParamTys...),
                                      PILType T,
                                      const TypeInfo &type,
                                      ArgTys... args) {
   auto operation = makeLambdaDynamicPackingOperation<ResultTy>(
      [&](IRGenFunction &IGF, PILType T, const TypeInfo &type,
          FixedPacking packing) {
         return fn(IGF, args..., T, type, packing);
      });
   emitDynamicPackingOperation(IGF, T, type, operation);
   return operation.get(IGF, T, type);
}

/// Emit a 'projectBuffer' operation.  Always returns a T*.
static Address emitDefaultProjectBuffer(IRGenFunction &IGF, Address buffer,
                                        PILType T, const TypeInfo &type,
                                        FixedPacking packing) {
   llvm::PointerType *resultTy = type.getStorageType()->getPointerTo();
   switch (packing) {
      case FixedPacking::Allocate: {

         // Use copy-on-write existentials?
         auto &IGM = IGF.IGM;
         auto &Builder = IGF.Builder;
         Address boxAddress(
            Builder.CreateBitCast(buffer.getAddress(),
                                  IGM.RefCountedPtrTy->getPointerTo()),
            buffer.getAlignment());
         auto *boxStart = IGF.Builder.CreateLoad(boxAddress);
         auto *alignmentMask = type.getAlignmentMask(IGF, T);
         auto *heapHeaderSize = llvm::ConstantInt::get(
            IGM.SizeTy, IGM.RefCountedStructSize.getValue());
         auto *startOffset =
            Builder.CreateAnd(Builder.CreateAdd(heapHeaderSize, alignmentMask),
                              Builder.CreateNot(alignmentMask));
         auto *addressInBox =
            IGF.emitByteOffsetGEP(boxStart, startOffset, IGM.OpaqueTy);

         addressInBox = Builder.CreateBitCast(addressInBox, resultTy);
         return type.getAddressForPointer(addressInBox);
      }

      case FixedPacking::OffsetZero: {
         return IGF.Builder.CreateBitCast(buffer, resultTy, "object");
      }

      case FixedPacking::Dynamic:
         return emitForDynamicPacking(IGF, &emitDefaultProjectBuffer,
                                      T, type, buffer);

   }
   llvm_unreachable("bad packing!");

}

/// Emit an 'allocateBuffer' operation.  Always returns a T*.
static Address emitDefaultAllocateBuffer(IRGenFunction &IGF, Address buffer,
                                         PILType T, const TypeInfo &type,
                                         FixedPacking packing) {
   switch (packing) {
      case FixedPacking::Allocate: {
         llvm::Value *box, *address;
         auto *metadata = IGF.emitTypeMetadataRefForLayout(T);
         IGF.emitAllocBoxCall(metadata, box, address);
         IGF.Builder.CreateStore(
            box, Address(IGF.Builder.CreateBitCast(buffer.getAddress(),
                                                   box->getType()->getPointerTo()),
                         buffer.getAlignment()));

         llvm::PointerType *resultTy = type.getStorageType()->getPointerTo();
         address = IGF.Builder.CreateBitCast(address, resultTy);
         return type.getAddressForPointer(address);
      }

      case FixedPacking::OffsetZero:
         return emitDefaultProjectBuffer(IGF, buffer, T, type, packing);

      case FixedPacking::Dynamic:
         return emitForDynamicPacking(IGF, &emitDefaultAllocateBuffer,
                                      T, type, buffer);
   }
   llvm_unreachable("bad packing!");
}

/// Emit an 'initializeBufferWithCopyOfBuffer' operation.
/// Returns the address of the destination object.
static Address emitDefaultInitializeBufferWithCopyOfBuffer(
   IRGenFunction &IGF, Address destBuffer, Address srcBuffer, PILType T,
   const TypeInfo &type, FixedPacking packing) {
   // Special-case dynamic packing in order to thread the jumps.
   if (packing == FixedPacking::Dynamic)
      return emitForDynamicPacking(IGF,
                                   &emitDefaultInitializeBufferWithCopyOfBuffer,
                                   T, type, destBuffer, srcBuffer);

   if (packing == FixedPacking::OffsetZero) {
      Address destObject =
         emitDefaultAllocateBuffer(IGF, destBuffer, T, type, packing);
      Address srcObject =
         emitDefaultProjectBuffer(IGF, srcBuffer, T, type, packing);
      type.initializeWithCopy(IGF, destObject, srcObject, T, true);
      return destObject;
   } else {
      assert(packing == FixedPacking::Allocate);
      auto *destReferenceAddr = IGF.Builder.CreateBitCast(
         destBuffer.getAddress(), IGF.IGM.RefCountedPtrTy->getPointerTo());
      auto *srcReferenceAddr = IGF.Builder.CreateBitCast(
         srcBuffer.getAddress(), IGF.IGM.RefCountedPtrTy->getPointerTo());
      auto *srcReference =
         IGF.Builder.CreateLoad(srcReferenceAddr, srcBuffer.getAlignment());
      IGF.emitNativeStrongRetain(srcReference, IGF.getDefaultAtomicity());
      IGF.Builder.CreateStore(
         srcReference, Address(destReferenceAddr, destBuffer.getAlignment()));
      return emitDefaultProjectBuffer(IGF, destBuffer, T, type, packing);
   }
}

// Metaprogram some of the common boilerplate here:
//   - the default implementation in TypeInfo
//   - the value-witness emitter which tries to avoid some dynamic
//     dispatch and the recomputation of the fixed packing

#define DEFINE_BINARY_BUFFER_OP(LOWER, TITLE)                             \
Address TypeInfo::LOWER(IRGenFunction &IGF, Address dest, Address src,    \
                        PILType T) const {                                \
  return emitDefault##TITLE(IGF, dest, src, T, *this,                     \
                            getFixedPacking(IGF.IGM));                    \
}                                                                         \
static Address emit##TITLE(IRGenFunction &IGF, Address dest, Address src, \
                           PILType T, const TypeInfo &type,               \
                           FixedPacking packing) {                        \
  if (packing == FixedPacking::Dynamic)                                   \
    return type.LOWER(IGF, dest, src, T);                                 \
  return emitDefault##TITLE(IGF, dest, src, T, type, packing);            \
}
DEFINE_BINARY_BUFFER_OP(initializeBufferWithCopyOfBuffer,
                        InitializeBufferWithCopyOfBuffer)
#undef DEFINE_BINARY_BUFFER_OP


static llvm::Value *getArg(llvm::Function::arg_iterator &it,
                           StringRef name) {
   llvm::Value *arg = &*(it++);
   arg->setName(name);
   return arg;
}

/// Get the next argument as a pointer to the given storage type.
static Address getArgAs(IRGenFunction &IGF,
                        llvm::Function::arg_iterator &it,
                        const TypeInfo &type,
                        StringRef name) {
   llvm::Value *arg = getArg(it, name);
   llvm::Value *result =
      IGF.Builder.CreateBitCast(arg, type.getStorageType()->getPointerTo());
   return type.getAddressForPointer(result);
}

/// Get the next argument as a pointer to the given storage type.
static Address getArgAsBuffer(IRGenFunction &IGF,
                              llvm::Function::arg_iterator &it,
                              StringRef name) {
   llvm::Value *arg = getArg(it, name);
   return Address(arg, getFixedBufferAlignment(IGF.IGM));
}

static CanType getFormalTypeInContext(CanType abstractType, DeclContext *dc) {
   // Map the parent of any non-generic nominal type.
   if (auto nominalType = dyn_cast<NominalType>(abstractType)) {
      // If it doesn't have a parent, or the parent doesn't need remapping,
      // do nothing.
      auto abstractParentType = nominalType.getParent();
      if (!abstractParentType) return abstractType;
      auto parentType = getFormalTypeInContext(abstractParentType, dc);
      if (abstractParentType == parentType) return abstractType;

      // Otherwise, rebuild the type.
      return CanType(NominalType::get(nominalType->getDecl(), parentType,
                                      nominalType->getDecl()->getAstContext()));

      // Map unbound types into their defining context.
   } else if (auto ugt = dyn_cast<UnboundGenericType>(abstractType)) {
      return dc->mapTypeIntoContext(ugt->getDecl()->getDeclaredInterfaceType())
         ->getCanonicalType();

      // Everything else stays the same.
   } else {
      return abstractType;
   }
}

/// Given an abstract type --- a type possibly expressed in terms of
/// unbound generic types --- return the formal type within the type's
/// primary defining context.
static CanType getFormalTypeInContext(CanType abstractType) {
   if (auto nominal = abstractType.getAnyNominal())
      return getFormalTypeInContext(abstractType, nominal);
   return abstractType;
}

void irgen::getArgAsLocalSelfTypeMetadata(IRGenFunction &IGF, llvm::Value *arg,
                                          CanType abstractType) {
   assert(arg->getType() == IGF.IGM.TypeMetadataPtrTy &&
          "Self argument is not a type?!");

   auto formalType = getFormalTypeInContext(abstractType);
   IGF.bindLocalTypeDataFromTypeMetadata(formalType, IsExact, arg,
                                         MetadataState::Complete);
}

/// Get the next argument and use it as the 'self' type metadata.
static void getArgAsLocalSelfTypeMetadata(IRGenFunction &IGF,
                                          llvm::Function::arg_iterator &it,
                                          CanType abstractType) {
   llvm::Value *arg = &*it++;
   getArgAsLocalSelfTypeMetadata(IGF, arg, abstractType);
}

/// Build a specific value-witness function.
static void buildValueWitnessFunction(IRGenModule &IGM,
                                      llvm::Function *fn,
                                      ValueWitness index,
                                      FixedPacking packing,
                                      CanType abstractType,
                                      PILType concreteType,
                                      const TypeInfo &type) {
   assert(isValueWitnessFunction(index));

   IRGenFunction IGF(IGM, fn);
   if (IGM.DebugInfo)
      IGM.DebugInfo->emitArtificialFunction(IGF, fn);

   auto argv = fn->arg_begin();
   switch (index) {
      case ValueWitness::AssignWithCopy: {
         Address dest = getArgAs(IGF, argv, type, "dest");
         Address src = getArgAs(IGF, argv, type, "src");
         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);
         type.assignWithCopy(IGF, dest, src, concreteType, true);
         dest = IGF.Builder.CreateBitCast(dest, IGF.IGM.OpaquePtrTy);
         IGF.Builder.CreateRet(dest.getAddress());
         return;
      }

      case ValueWitness::AssignWithTake: {
         Address dest = getArgAs(IGF, argv, type, "dest");
         Address src = getArgAs(IGF, argv, type, "src");
         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);
         type.assignWithTake(IGF, dest, src, concreteType, true);
         dest = IGF.Builder.CreateBitCast(dest, IGF.IGM.OpaquePtrTy);
         IGF.Builder.CreateRet(dest.getAddress());
         return;
      }

      case ValueWitness::Destroy: {
         Address object = getArgAs(IGF, argv, type, "object");
         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);
         type.destroy(IGF, object, concreteType, true);
         IGF.Builder.CreateRetVoid();
         return;
      }

      case ValueWitness::InitializeBufferWithCopyOfBuffer: {
         Address dest = getArgAsBuffer(IGF, argv, "dest");
         Address src = getArgAsBuffer(IGF, argv, "src");
         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);

         Address result =
            emitInitializeBufferWithCopyOfBuffer(IGF, dest, src, concreteType,
                                                 type, packing);
         result = IGF.Builder.CreateBitCast(result, IGF.IGM.OpaquePtrTy);
         IGF.Builder.CreateRet(result.getAddress());
         return;
      }

      case ValueWitness::InitializeWithCopy: {
         Address dest = getArgAs(IGF, argv, type, "dest");
         Address src = getArgAs(IGF, argv, type, "src");
         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);

         type.initializeWithCopy(IGF, dest, src, concreteType, true);
         dest = IGF.Builder.CreateBitCast(dest, IGF.IGM.OpaquePtrTy);
         IGF.Builder.CreateRet(dest.getAddress());
         return;
      }

      case ValueWitness::InitializeWithTake: {
         Address dest = getArgAs(IGF, argv, type, "dest");
         Address src = getArgAs(IGF, argv, type, "src");
         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);

         type.initializeWithTake(IGF, dest, src, concreteType, true);
         dest = IGF.Builder.CreateBitCast(dest, IGF.IGM.OpaquePtrTy);
         IGF.Builder.CreateRet(dest.getAddress());
         return;
      }

      case ValueWitness::GetEnumTag: {
         auto &strategy = getEnumImplStrategy(IGM, concreteType);

         llvm::Value *value = getArg(argv, "value");
         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);

         auto enumTy = type.getStorageType()->getPointerTo();
         value = IGF.Builder.CreateBitCast(value, enumTy);
         auto enumAddr = type.getAddressForPointer(value);

         llvm::Value *result = strategy.emitGetEnumTag(IGF, concreteType, enumAddr);
         IGF.Builder.CreateRet(result);
         return;
      }

      case ValueWitness::DestructiveProjectEnumData: {
         auto &strategy = getEnumImplStrategy(IGM, concreteType);

         llvm::Value *value = getArg(argv, "value");
         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);

         if (!strategy.getElementsWithPayload().empty()) {
            strategy.destructiveProjectDataForLoad(
               IGF, concreteType,
               Address(value, type.getBestKnownAlignment()));
         }

         IGF.Builder.CreateRetVoid();
         return;
      }

      case ValueWitness::DestructiveInjectEnumTag: {
         auto &strategy = getEnumImplStrategy(IGM, concreteType);

         llvm::Value *value = getArg(argv, "value");

         auto enumTy = type.getStorageType()->getPointerTo();
         value = IGF.Builder.CreateBitCast(value, enumTy);

         llvm::Value *tag = getArg(argv, "tag");

         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);

         strategy.emitStoreTag(IGF, concreteType,
                               Address(value, type.getBestKnownAlignment()),
                               tag);

         IGF.Builder.CreateRetVoid();
         return;
      }

      case ValueWitness::GetEnumTagSinglePayload: {
         llvm::Value *value = getArg(argv, "value");
         auto enumTy = type.getStorageType()->getPointerTo();
         value = IGF.Builder.CreateBitCast(value, enumTy);

         llvm::Value *numEmptyCases = getArg(argv, "numEmptyCases");

         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);

         llvm::Value *idx = type.getEnumTagSinglePayload(
            IGF, numEmptyCases, Address(value, type.getBestKnownAlignment()),
            concreteType, true);
         IGF.Builder.CreateRet(idx);
         return;
      }

      case ValueWitness::StoreEnumTagSinglePayload: {
         llvm::Value *value = getArg(argv, "value");
         auto enumTy = type.getStorageType()->getPointerTo();
         value = IGF.Builder.CreateBitCast(value, enumTy);

         llvm::Value *whichCase = getArg(argv, "whichCase");
         llvm::Value *numEmptyCases = getArg(argv, "numEmptyCases");

         getArgAsLocalSelfTypeMetadata(IGF, argv, abstractType);

         type.storeEnumTagSinglePayload(IGF, whichCase, numEmptyCases,
                                        Address(value, type.getBestKnownAlignment()),
                                        concreteType, true);
         IGF.Builder.CreateRetVoid();
         return;
      }

      case ValueWitness::Size:
      case ValueWitness::Flags:
      case ValueWitness::ExtraInhabitantCount:
      case ValueWitness::Stride:
         llvm_unreachable("these value witnesses aren't functions");
   }
   llvm_unreachable("bad value witness kind!");
}

/// Return a function which takes two pointer arguments and returns
/// void immediately.
static llvm::Constant *getNoOpVoidFunction(IRGenModule &IGM) {
   llvm::Type *argTys[] = { IGM.Int8PtrTy, IGM.TypeMetadataPtrTy };
   return IGM.getOrCreateHelperFunction("__swift_noop_void_return",
                                        IGM.VoidTy, argTys,
                                        [&](IRGenFunction &IGF) {
                                           IGF.Builder.CreateRetVoid();
                                        });
}

/// Return a function which takes three pointer arguments and does a
/// retaining assignWithCopy on the first two: it loads a pointer from
/// the second, retains it, loads a pointer from the first, stores the
/// new pointer in the first, and releases the old pointer.
static llvm::Constant *getAssignWithCopyStrongFunction(IRGenModule &IGM) {
   llvm::Type *ptrPtrTy = IGM.RefCountedPtrTy->getPointerTo();
   llvm::Type *argTys[] = { ptrPtrTy, ptrPtrTy, IGM.WitnessTablePtrTy };
   return IGM.getOrCreateHelperFunction("__swift_assignWithCopy_strong",
                                        ptrPtrTy, argTys,
                                        [&](IRGenFunction &IGF) {
                                           auto it = IGF.CurFn->arg_begin();
                                           Address dest(&*(it++), IGM.getPointerAlignment());
                                           Address src(&*(it++), IGM.getPointerAlignment());

                                           llvm::Value *newValue = IGF.Builder.CreateLoad(src, "new");
                                           IGF.emitNativeStrongRetain(newValue, IGF.getDefaultAtomicity());
                                           llvm::Value *oldValue = IGF.Builder.CreateLoad(dest, "old");
                                           IGF.Builder.CreateStore(newValue, dest);
                                           IGF.emitNativeStrongRelease(oldValue, IGF.getDefaultAtomicity());

                                           IGF.Builder.CreateRet(dest.getAddress());
                                        });
}

/// Return a function which takes three pointer arguments and does a
/// retaining assignWithTake on the first two: it loads a pointer from
/// the second, retains it, loads a pointer from the first, stores the
/// new pointer in the first, and releases the old pointer.
static llvm::Constant *getAssignWithTakeStrongFunction(IRGenModule &IGM) {
   llvm::Type *ptrPtrTy = IGM.RefCountedPtrTy->getPointerTo();
   llvm::Type *argTys[] = { ptrPtrTy, ptrPtrTy, IGM.WitnessTablePtrTy };
   return IGM.getOrCreateHelperFunction("__swift_assignWithTake_strong",
                                        ptrPtrTy, argTys,
                                        [&](IRGenFunction &IGF) {
                                           auto it = IGF.CurFn->arg_begin();
                                           Address dest(&*(it++), IGM.getPointerAlignment());
                                           Address src(&*(it++), IGM.getPointerAlignment());

                                           llvm::Value *newValue = IGF.Builder.CreateLoad(src, "new");
                                           llvm::Value *oldValue = IGF.Builder.CreateLoad(dest, "old");
                                           IGF.Builder.CreateStore(newValue, dest);
                                           IGF.emitNativeStrongRelease(oldValue, IGF.getDefaultAtomicity());

                                           IGF.Builder.CreateRet(dest.getAddress());
                                        });
}

/// Return a function which takes three pointer arguments and does a
/// retaining initWithCopy on the first two: it loads a pointer from
/// the second, retains it, and stores that in the first.
static llvm::Constant *getInitWithCopyStrongFunction(IRGenModule &IGM) {
   llvm::Type *ptrPtrTy = IGM.RefCountedPtrTy->getPointerTo();
   llvm::Type *argTys[] = { ptrPtrTy, ptrPtrTy, IGM.WitnessTablePtrTy };
   return IGM.getOrCreateHelperFunction("__swift_initWithCopy_strong",
                                        ptrPtrTy, argTys,
                                        [&](IRGenFunction &IGF) {
                                           auto it = IGF.CurFn->arg_begin();
                                           Address dest(&*(it++), IGM.getPointerAlignment());
                                           Address src(&*(it++), IGM.getPointerAlignment());

                                           llvm::Value *newValue = IGF.Builder.CreateLoad(src, "new");
                                           IGF.emitNativeStrongRetain(newValue, IGF.getDefaultAtomicity());
                                           IGF.Builder.CreateStore(newValue, dest);

                                           IGF.Builder.CreateRet(dest.getAddress());
                                        });
}

/// Return a function which takes two pointer arguments, loads a
/// pointer from the first, and calls swift_release on it immediately.
static llvm::Constant *getDestroyStrongFunction(IRGenModule &IGM) {
   llvm::Type *argTys[] = { IGM.Int8PtrPtrTy, IGM.WitnessTablePtrTy };
   return IGM.getOrCreateHelperFunction("__swift_destroy_strong",
                                        IGM.VoidTy, argTys,
                                        [&](IRGenFunction &IGF) {
                                           Address arg(&*IGF.CurFn->arg_begin(), IGM.getPointerAlignment());
                                           IGF.emitNativeStrongRelease(IGF.Builder.CreateLoad(arg), IGF.getDefaultAtomicity());
                                           IGF.Builder.CreateRetVoid();
                                        });
}

/// Return a function which takes two pointer arguments, memcpys
/// from the second to the first, and returns the first argument.
static llvm::Constant *getMemCpyFunction(IRGenModule &IGM,
                                         const TypeInfo &objectTI) {
   // If we don't have a fixed type, use the standard copy-opaque-POD
   // routine.  It's not quite clear how in practice we'll be able to
   // conclude that something is known-POD without knowing its size,
   // but it's (1) conceivable and (2) needed as a general export anyway.
   auto *fixedTI = dyn_cast<FixedTypeInfo>(&objectTI);
   if (!fixedTI) return IGM.getCopyPODFn();

   // We need to unique by both size and alignment.  Note that we're
   // assuming that it's safe to call a function that returns a pointer
   // at a site that assumes the function returns void.
   llvm::SmallString<40> name;
   {
      llvm::raw_svector_ostream nameStream(name);
      nameStream << "__swift_memcpy";
      nameStream << fixedTI->getFixedSize().getValue();
      nameStream << '_';
      nameStream << fixedTI->getFixedAlignment().getValue();
   }

   llvm::Type *argTys[] = { IGM.Int8PtrTy, IGM.Int8PtrTy, IGM.TypeMetadataPtrTy };
   return IGM.getOrCreateHelperFunction(name, IGM.Int8PtrTy, argTys,
                                        [&](IRGenFunction &IGF) {
                                           auto it = IGF.CurFn->arg_begin();
                                           Address dest(&*it++, fixedTI->getFixedAlignment());
                                           Address src(&*it++, fixedTI->getFixedAlignment());
                                           IGF.emitMemCpy(dest, src, fixedTI->getFixedSize());
                                           IGF.Builder.CreateRet(dest.getAddress());
                                        });
}

/// Find a witness to the fact that a type is a value type.
/// Always adds an i8*.
static void addValueWitness(IRGenModule &IGM,
                            ConstantStructBuilder &B,
                            ValueWitness index,
                            FixedPacking packing,
                            CanType abstractType,
                            PILType concreteType,
                            const TypeInfo &concreteTI) {
   auto addFunction = [&](llvm::Constant *fn) {
      B.addBitCast(fn, IGM.Int8PtrTy);
   };

   // Try to use a standard function.
   switch (index) {
      case ValueWitness::Destroy:
         if (concreteTI.isPOD(ResilienceExpansion::Maximal)) {
            return addFunction(getNoOpVoidFunction(IGM));
         } else if (concreteTI.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal)) {
            return addFunction(getDestroyStrongFunction(IGM));
         }
         goto standard;

      case ValueWitness::InitializeBufferWithCopyOfBuffer:
         if (packing == FixedPacking::OffsetZero) {
            if (concreteTI.isPOD(ResilienceExpansion::Maximal)) {
               return addFunction(getMemCpyFunction(IGM, concreteTI));
            } else if (concreteTI.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal)) {
               return addFunction(getInitWithCopyStrongFunction(IGM));
            }
         }
         goto standard;

      case ValueWitness::InitializeWithTake:
         if (concreteTI.isBitwiseTakable(ResilienceExpansion::Maximal)) {
            return addFunction(getMemCpyFunction(IGM, concreteTI));
         }
         goto standard;

      case ValueWitness::AssignWithCopy:
         if (concreteTI.isPOD(ResilienceExpansion::Maximal)) {
            return addFunction(getMemCpyFunction(IGM, concreteTI));
         } else if (concreteTI.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal)) {
            return addFunction(getAssignWithCopyStrongFunction(IGM));
         }
         goto standard;

      case ValueWitness::AssignWithTake:
         if (concreteTI.isPOD(ResilienceExpansion::Maximal)) {
            return addFunction(getMemCpyFunction(IGM, concreteTI));
         } else if (concreteTI.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal)) {
            return addFunction(getAssignWithTakeStrongFunction(IGM));
         }
         goto standard;

      case ValueWitness::InitializeWithCopy:
         if (concreteTI.isPOD(ResilienceExpansion::Maximal)) {
            return addFunction(getMemCpyFunction(IGM, concreteTI));
         } else if (concreteTI.isSingleSwiftRetainablePointer(ResilienceExpansion::Maximal)) {
            return addFunction(getInitWithCopyStrongFunction(IGM));
         }
         goto standard;

      case ValueWitness::Size: {
         if (auto value = concreteTI.getStaticSize(IGM))
            return B.add(value);

         // Just fill in 0 here if the type can't be statically laid out.
         return B.addSize(Size(0));
      }

      case ValueWitness::Flags: {
         ValueWitnessFlags flags;

         // If we locally know that the type has fixed layout, we can emit
         // meaningful flags for it.
         if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&concreteTI)) {
            assert(packing == FixedPacking::OffsetZero ||
                   packing == FixedPacking::Allocate);
            bool isInline = packing == FixedPacking::OffsetZero;
            bool isBitwiseTakable =
               fixedTI->isBitwiseTakable(ResilienceExpansion::Maximal);
            assert(isBitwiseTakable || !isInline);
            flags = flags.withAlignment(fixedTI->getFixedAlignment().getValue())
               .withPOD(fixedTI->isPOD(ResilienceExpansion::Maximal))
               .withInlineStorage(isInline)
               .withBitwiseTakable(isBitwiseTakable);
         } else {
            flags = flags.withIncomplete(true);
         }

         if (concreteType.getEnumOrBoundGenericEnum())
            flags = flags.withEnumWitnesses(true);

         return B.addInt32(flags.getOpaqueValue());
      }

      case ValueWitness::ExtraInhabitantCount: {
         unsigned value = 0;
         if (auto *fixedTI = dyn_cast<FixedTypeInfo>(&concreteTI)) {
            value = fixedTI->getFixedExtraInhabitantCount(IGM);
         }
         return B.addInt32(value);
      }

      case ValueWitness::Stride: {
         if (auto value = concreteTI.getStaticStride(IGM))
            return B.add(value);

         // Just fill in null here if the type can't be statically laid out.
         return B.addSize(Size(0));
      }

      case ValueWitness::GetEnumTagSinglePayload:
      case ValueWitness::StoreEnumTagSinglePayload: {
         goto standard;
      }

      case ValueWitness::GetEnumTag:
      case ValueWitness::DestructiveProjectEnumData:
      case ValueWitness::DestructiveInjectEnumTag:
         assert(concreteType.getEnumOrBoundGenericEnum());
         goto standard;
   }
   llvm_unreachable("bad value witness kind");

standard:
   llvm::Function *fn =
      IGM.getAddrOfValueWitness(abstractType, index, ForDefinition);
   if (fn->empty())
      buildValueWitnessFunction(IGM, fn, index, packing, abstractType,
                                concreteType, concreteTI);
   addFunction(fn);
}

static bool shouldAddEnumWitnesses(CanType abstractType) {
   // Needs to handle UnboundGenericType.
   return dyn_cast_or_null<EnumDecl>(abstractType.getAnyNominal()) != nullptr;
}

static llvm::StructType *getValueWitnessTableType(IRGenModule &IGM,
                                                  CanType abstractType) {
   return shouldAddEnumWitnesses(abstractType)
          ? IGM.getEnumValueWitnessTableTy()
          : IGM.getValueWitnessTableTy();
}

/// Collect the value witnesses for a particular type.
static void addValueWitnesses(IRGenModule &IGM,
                              ConstantStructBuilder &B,
                              FixedPacking packing,
                              CanType abstractType,
                              PILType concreteType,
                              const TypeInfo &concreteTI) {
   for (unsigned i = 0; i != NumRequiredValueWitnesses; ++i) {
      addValueWitness(IGM, B, ValueWitness(i), packing,
                      abstractType, concreteType, concreteTI);
   }
   if (shouldAddEnumWitnesses(abstractType)) {
      for (auto i = unsigned(ValueWitness::First_EnumValueWitness);
           i <= unsigned(ValueWitness::Last_EnumValueWitness);
           ++i) {
         addValueWitness(IGM, B, ValueWitness(i), packing,
                         abstractType, concreteType, concreteTI);
      }
   }
}

/// True if a type has a generic-parameter-dependent value witness table.
/// Currently, this is true if the size and/or alignment of the type is
/// dependent on its generic parameters.
bool irgen::hasDependentValueWitnessTable(IRGenModule &IGM, CanType ty) {
   return !IGM.getTypeInfoForUnlowered(getFormalTypeInContext(ty)).isFixedSize();
}

static void addValueWitnessesForAbstractType(IRGenModule &IGM,
                                             ConstantStructBuilder &B,
                                             CanType abstractType,
                                             bool &canBeConstant) {
   CanType concreteFormalType = getFormalTypeInContext(abstractType);

   auto concreteLoweredType = IGM.getLoweredType(concreteFormalType);
   auto &concreteTI = IGM.getTypeInfo(concreteLoweredType);
   FixedPacking packing = concreteTI.getFixedPacking(IGM);

   // For now, assume that we never have any interest in dynamically
   // changing the value witnesses for something that's fixed-layout.
   canBeConstant = concreteTI.isFixedSize();

   addValueWitnesses(IGM, B, packing, abstractType,
                     concreteLoweredType, concreteTI);
}

static constexpr uint64_t sizeAndAlignment(Size size, Alignment alignment) {
   return ((uint64_t)size.getValue() << 32) | alignment.getValue();
}

/// Return a reference to a known value witness table from the runtime
/// that's suitable for the given type, if there is one, or return null
/// if we should emit a new one.
static ConstantReference
getAddrOfKnownValueWitnessTable(IRGenModule &IGM, CanType type,
                                bool relativeReference) {
   // Native PE binaries shouldn't reference data symbols across DLLs, so disable
   // this on Windows, unless we're forming a relative indirectable reference.
   if (IGM.useDllStorage() && !relativeReference)
      return {};

   if (auto nom = type->getAnyNominal()) {
      // TODO: Non-C enums have extra inhabitants and also need additional value
      // witnesses for their tag manipulation (except when they're empty, in
      // which case values never exist to witness).
      if (auto enumDecl = dyn_cast<EnumDecl>(nom))
         if (/*!enumDecl->isObjC() && */!type->isUninhabited())
            return {};
   }

   auto &C = IGM.Context;

   type = getFormalTypeInContext(type);

   auto &ti = IGM.getTypeInfoForUnlowered(AbstractionPattern::getOpaque(), type);

   // We only have witnesses for fixed type info.
   auto *fixedTI = dyn_cast<FixedTypeInfo>(&ti);
   if (!fixedTI)
      return {};

   CanType witnessSurrogate;
   ReferenceCounting refCounting;

   // Empty types can use empty tuple witnesses.
   if (ti.isKnownEmpty(ResilienceExpansion::Maximal)) {
      witnessSurrogate = TupleType::getEmpty(C);
      // Handle common POD type layouts.
   } else if (fixedTI->isPOD(ResilienceExpansion::Maximal)
              && fixedTI->getFixedExtraInhabitantCount(IGM) == 0) {
      // Reuse one of the integer witnesses if applicable.
      switch (sizeAndAlignment(fixedTI->getFixedSize(),
                               fixedTI->getFixedAlignment())) {
         case sizeAndAlignment(Size(0), Alignment(1)):
            witnessSurrogate = TupleType::getEmpty(C);
            break;
         case sizeAndAlignment(Size(1), Alignment(1)):
            witnessSurrogate = BuiltinIntegerType::get(8, C)->getCanonicalType();
            break;
         case sizeAndAlignment(Size(2), Alignment(2)):
            witnessSurrogate = BuiltinIntegerType::get(16, C)->getCanonicalType();
            break;
         case sizeAndAlignment(Size(4), Alignment(4)):
            witnessSurrogate = BuiltinIntegerType::get(32, C)->getCanonicalType();
            break;
         case sizeAndAlignment(Size(8), Alignment(8)):
            witnessSurrogate = BuiltinIntegerType::get(64, C)->getCanonicalType();
            break;
         case sizeAndAlignment(Size(16), Alignment(16)):
            witnessSurrogate = BuiltinIntegerType::get(128, C)->getCanonicalType();
            break;
         case sizeAndAlignment(Size(32), Alignment(32)):
            witnessSurrogate = BuiltinIntegerType::get(256, C)->getCanonicalType();
            break;
         case sizeAndAlignment(Size(64), Alignment(64)):
            witnessSurrogate = BuiltinIntegerType::get(512, C)->getCanonicalType();
            break;
      }
   } else if (fixedTI->isSingleRetainablePointer(ResilienceExpansion::Maximal,
                                                 &refCounting)) {
      switch (refCounting) {
         case ReferenceCounting::Native:
            witnessSurrogate = C.TheNativeObjectType;
            break;
//         case ReferenceCounting::ObjC:
         case ReferenceCounting::Block:
         case ReferenceCounting::Unknown:
            witnessSurrogate = C.getAnyObjectType();
            break;
         case ReferenceCounting::Bridge:
            witnessSurrogate = C.TheBridgeObjectType;
            break;
         case ReferenceCounting::Error:
            break;
      }
   }

   if (witnessSurrogate) {
      if (relativeReference) {
         return IGM.getAddrOfLLVMVariableOrGOTEquivalent(
            LinkEntity::forValueWitnessTable(witnessSurrogate));
      } else {
         return {IGM.getAddrOfValueWitnessTable(witnessSurrogate),
                 ConstantReference::Direct};
      }
   }
   return {};
}

/// Emit a value-witness table for the given type.
ConstantReference irgen::emitValueWitnessTable(IRGenModule &IGM,
                                               CanType abstractType,
                                               bool isPattern,
                                               bool relativeReference) {
   // We shouldn't emit global value witness tables for generic type instances.
   assert(!isa<BoundGenericType>(abstractType) &&
          "emitting VWT for generic instance");

   // See if we can use a prefab witness table from the runtime.
   if (!isPattern) {
      if (auto known = getAddrOfKnownValueWitnessTable(IGM, abstractType,
                                                       relativeReference)) {
         return known;
      }
   }

   // We should never be making a pattern if the layout isn't fixed.
   // The reverse can be true for types whose layout depends on
   // resilient types.
   assert((!isPattern || hasDependentValueWitnessTable(IGM, abstractType)) &&
          "emitting VWT pattern for fixed-layout type");

   ConstantInitBuilder builder(IGM);
   auto witnesses =
      builder.beginStruct(getValueWitnessTableType(IGM, abstractType));

   bool canBeConstant = false;
   addValueWitnessesForAbstractType(IGM, witnesses, abstractType, canBeConstant);

   // If this is just an instantiation pattern, we should never be modifying
   // it in-place.
   if (isPattern) canBeConstant = true;

   auto addr = IGM.getAddrOfValueWitnessTable(abstractType,
                                              witnesses.finishAndCreateFuture());
   auto global = cast<llvm::GlobalVariable>(addr);
   global->setConstant(canBeConstant);

   return {llvm::ConstantExpr::getBitCast(global, IGM.WitnessTablePtrTy),
           ConstantReference::Direct};
}

llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
                                                 const FixedTypeInfo &ti) {
   auto silTy = PILType::getPrimitiveAddressType(t);
   // Collect the interesting information that gets encoded in a type layout
   // record, to see if there's one we can reuse.
   unsigned size = ti.getFixedSize().getValue();
   unsigned align = ti.getFixedAlignment().getValue();

   bool pod = ti.isPOD(ResilienceExpansion::Maximal);
   bool bt = ti.isBitwiseTakable(ResilienceExpansion::Maximal);
   unsigned numExtraInhabitants = ti.getFixedExtraInhabitantCount(*this);

   // Try to use common type layouts exported by the runtime.
   llvm::Constant *commonValueWitnessTable = nullptr;
   if (pod && bt && numExtraInhabitants == 0) {
      if (size == 0)
         commonValueWitnessTable =
            getAddrOfValueWitnessTable(Context.TheEmptyTupleType);
      if (   (size ==  1 && align ==  1)
             || (size ==  2 && align ==  2)
             || (size ==  4 && align ==  4)
             || (size ==  8 && align ==  8)
             || (size == 16 && align == 16)
             || (size == 32 && align == 32))
         commonValueWitnessTable =
            getAddrOfValueWitnessTable(BuiltinIntegerType::get(size * 8, Context)
                                          ->getCanonicalType());
   }

   if (commonValueWitnessTable) {
      auto index = llvm::ConstantInt::get(Int32Ty,
                                          (unsigned)ValueWitness::First_TypeLayoutWitness);
      return llvm::ConstantExpr::getGetElementPtr(Int8PtrTy,
                                                  commonValueWitnessTable,
                                                  index);
   }

   // Otherwise, see if a layout has been emitted with these characteristics
   // already.
   FixedLayoutKey key{size, numExtraInhabitants, align, pod, bt};

   auto found = PrivateFixedLayouts.find(key);
   if (found != PrivateFixedLayouts.end())
      return found->second;

   // Emit the layout values.
   ConstantInitBuilder builder(*this);
   auto witnesses = builder.beginStruct();
   FixedPacking packing = ti.getFixedPacking(*this);
   for (auto witness = ValueWitness::First_TypeLayoutWitness;
        witness <= ValueWitness::Last_RequiredTypeLayoutWitness;
        witness = ValueWitness(unsigned(witness) + 1)) {
      addValueWitness(*this, witnesses, witness, packing, t, silTy, ti);
   }

   auto layoutVar
      = witnesses.finishAndCreateGlobal(
         "type_layout_" + llvm::Twine(size)
         + "_" + llvm::Twine(align)
         + "_" + llvm::Twine::utohexstr(numExtraInhabitants)
         + (pod ? "_pod" :
            bt  ? "_bt"  : ""),
         getPointerAlignment(),
         /*constant*/ true,
         llvm::GlobalValue::PrivateLinkage);

   // Cast to the standard currency type for type layouts.
   auto layout = llvm::ConstantExpr::getBitCast(layoutVar, Int8PtrPtrTy);

   PrivateFixedLayouts.insert({key, layout});
   return layout;
}

FixedPacking TypeInfo::getFixedPacking(IRGenModule &IGM) const {
   auto fixedTI = dyn_cast<FixedTypeInfo>(this);

   // If the type isn't fixed, we have to do something dynamic.
   // FIXME: some types are provably too big (or aligned) to be
   // allocated inline.
   if (!fixedTI)
      return FixedPacking::Dynamic;

   // By convention we only store bitwise takable values inline.
   if (!fixedTI->isBitwiseTakable(ResilienceExpansion::Maximal))
      return FixedPacking::Allocate;

   Size bufferSize = getFixedBufferSize(IGM);
   Size requiredSize = fixedTI->getFixedSize();

   // Flat out, if we need more space than the buffer provides,
   // we always have to allocate.
   // FIXME: there might be some interesting cases where this
   // is suboptimal for enums.
   if (requiredSize > bufferSize)
      return FixedPacking::Allocate;

   Alignment bufferAlign = getFixedBufferAlignment(IGM);
   Alignment requiredAlign = fixedTI->getFixedAlignment();

   // If the buffer alignment is good enough for the type, great.
   if (bufferAlign >= requiredAlign)
      return FixedPacking::OffsetZero;

   // TODO: consider using a slower mode that dynamically checks
   // whether the buffer size is small enough.

   // Otherwise we're stuck and have to separately allocate.
   return FixedPacking::Allocate;
}

Address TypeInfo::indexArray(IRGenFunction &IGF, Address base,
                             llvm::Value *index, PILType T) const {
   // The stride of a Swift type may not match its LLVM size. If we know we have
   // a fixed stride different from our size, or we have a dynamic size,
   // do a byte-level GEP with the proper stride.
   const auto *fixedTI = dyn_cast<FixedTypeInfo>(this);

   llvm::Value *destValue = nullptr;
   Size stride(1);

   // TODO: Arrays currently lower-bound the stride to 1.
   if (!fixedTI || fixedTI->getFixedStride() != fixedTI->getFixedSize()) {
      llvm::Value *byteAddr = IGF.Builder.CreateBitCast(base.getAddress(),
                                                        IGF.IGM.Int8PtrTy);
      llvm::Value *size = getStride(IGF, T);
      if (size->getType() != index->getType())
         size = IGF.Builder.CreateZExtOrTrunc(size, index->getType());
      llvm::Value *distance = IGF.Builder.CreateNSWMul(index, size);
      destValue = IGF.Builder.CreateInBoundsGEP(byteAddr, distance);
      destValue = IGF.Builder.CreateBitCast(destValue, base.getType());
   } else {
      // We don't expose a non-inbounds GEP operation.
      destValue = IGF.Builder.CreateInBoundsGEP(base.getAddress(), index);
      stride = fixedTI->getFixedStride();
   }
   if (auto *IndexConst = dyn_cast<llvm::ConstantInt>(index)) {
      // If we know the indexing value, we can get a better guess on the
      // alignment.
      // This even works if the stride is not known (and assumed to be 1).
      stride *= IndexConst->getValue().getZExtValue();
   }
   Alignment Align = base.getAlignment().alignmentAtOffset(stride);
   return Address(destValue, Align);
}

Address TypeInfo::roundUpToTypeAlignment(IRGenFunction &IGF, Address base,
                                         PILType T) const {
   Alignment Align = base.getAlignment();
   llvm::Value *TyAlignMask = getAlignmentMask(IGF, T);
   if (auto *TyAlignMaskConst = dyn_cast<llvm::ConstantInt>(TyAlignMask)) {
      Alignment TyAlign(TyAlignMaskConst->getZExtValue() + 1);

      // No need to align if the base is already aligned.
      if (TyAlign <= Align)
         return base;
   }
   llvm::Value *Addr = base.getAddress();
   Addr = IGF.Builder.CreatePtrToInt(Addr, IGF.IGM.IntPtrTy);
   Addr = IGF.Builder.CreateNUWAdd(Addr, TyAlignMask);
   llvm::Value *InvertedMask = IGF.Builder.CreateNot(TyAlignMask);
   Addr = IGF.Builder.CreateAnd(Addr, InvertedMask);
   Addr = IGF.Builder.CreateIntToPtr(Addr, base.getAddress()->getType());
   return Address(Addr, Align);
}

void TypeInfo::destroyArray(IRGenFunction &IGF, Address array,
                            llvm::Value *count, PILType T) const {
   if (isPOD(ResilienceExpansion::Maximal))
      return;

   emitDestroyArrayCall(IGF, T, array, count);
}

void TypeInfo::initializeArrayWithCopy(IRGenFunction &IGF,
                                       Address dest, Address src,
                                       llvm::Value *count, PILType T) const {
   if (isPOD(ResilienceExpansion::Maximal)) {
      llvm::Value *stride = getStride(IGF, T);
      llvm::Value *byteCount = IGF.Builder.CreateNUWMul(stride, count);
      IGF.Builder.CreateMemCpy(dest.getAddress(),
                               dest.getAlignment().getValue(),
                               src.getAddress(),
                               src.getAlignment().getValue(), byteCount);
      return;
   }

   emitInitializeArrayWithCopyCall(IGF, T, dest, src, count);
}

void TypeInfo::initializeArrayWithTakeNoAlias(IRGenFunction &IGF, Address dest,
                                              Address src, llvm::Value *count,
                                              PILType T) const {
   if (isBitwiseTakable(ResilienceExpansion::Maximal)) {
      llvm::Value *stride = getStride(IGF, T);
      llvm::Value *byteCount = IGF.Builder.CreateNUWMul(stride, count);
      IGF.Builder.CreateMemCpy(dest.getAddress(),
                               dest.getAlignment().getValue(),
                               src.getAddress(),
                               src.getAlignment().getValue(), byteCount);
      return;
   }

   emitInitializeArrayWithTakeNoAliasCall(IGF, T, dest, src, count);
}

void TypeInfo::initializeArrayWithTakeFrontToBack(IRGenFunction &IGF,
                                                  Address dest, Address src,
                                                  llvm::Value *count, PILType T)
const {
   if (isBitwiseTakable(ResilienceExpansion::Maximal)) {
      llvm::Value *stride = getStride(IGF, T);
      llvm::Value *byteCount = IGF.Builder.CreateNUWMul(stride, count);
      IGF.Builder.CreateMemMove(dest.getAddress(), dest.getAlignment().getValue(),
                                src.getAddress(), src.getAlignment().getValue(),
                                byteCount);
      return;
   }

   emitInitializeArrayWithTakeFrontToBackCall(IGF, T, dest, src, count);
}

void TypeInfo::initializeArrayWithTakeBackToFront(IRGenFunction &IGF,
                                                  Address dest, Address src,
                                                  llvm::Value *count, PILType T)
const {
   if (isBitwiseTakable(ResilienceExpansion::Maximal)) {
      llvm::Value *stride = getStride(IGF, T);
      llvm::Value *byteCount = IGF.Builder.CreateNUWMul(stride, count);
      IGF.Builder.CreateMemMove(dest.getAddress(), dest.getAlignment().getValue(),
                                src.getAddress(), src.getAlignment().getValue(),
                                byteCount);
      return;
   }

   emitInitializeArrayWithTakeBackToFrontCall(IGF, T, dest, src, count);
}

void TypeInfo::assignArrayWithCopyNoAlias(IRGenFunction &IGF, Address dest,
                                          Address src, llvm::Value *count,
                                          PILType T) const {
   if (isPOD(ResilienceExpansion::Maximal)) {
      llvm::Value *stride = getStride(IGF, T);
      llvm::Value *byteCount = IGF.Builder.CreateNUWMul(stride, count);
      IGF.Builder.CreateMemCpy(dest.getAddress(),
                               dest.getAlignment().getValue(),
                               src.getAddress(),
                               src.getAlignment().getValue(), byteCount);
      return;
   }

   emitAssignArrayWithCopyNoAliasCall(IGF, T, dest, src, count);
}

void TypeInfo::assignArrayWithCopyFrontToBack(IRGenFunction &IGF, Address dest,
                                              Address src, llvm::Value *count,
                                              PILType T) const {
   if (isPOD(ResilienceExpansion::Maximal)) {
      llvm::Value *stride = getStride(IGF, T);
      llvm::Value *byteCount = IGF.Builder.CreateNUWMul(stride, count);
      IGF.Builder.CreateMemMove(dest.getAddress(), dest.getAlignment().getValue(),
                                src.getAddress(), src.getAlignment().getValue(),
                                byteCount);
      return;
   }

   emitAssignArrayWithCopyFrontToBackCall(IGF, T, dest, src, count);
}

void TypeInfo::assignArrayWithCopyBackToFront(IRGenFunction &IGF, Address dest,
                                              Address src, llvm::Value *count,
                                              PILType T) const {
   if (isPOD(ResilienceExpansion::Maximal)) {
      llvm::Value *stride = getStride(IGF, T);
      llvm::Value *byteCount = IGF.Builder.CreateNUWMul(stride, count);
      IGF.Builder.CreateMemMove(dest.getAddress(), dest.getAlignment().getValue(),
                                src.getAddress(), src.getAlignment().getValue(),
                                byteCount);
      return;
   }

   emitAssignArrayWithCopyBackToFrontCall(IGF, T, dest, src, count);
}

void TypeInfo::assignArrayWithTake(IRGenFunction &IGF, Address dest,
                                   Address src, llvm::Value *count,
                                   PILType T) const {
   if (isPOD(ResilienceExpansion::Maximal)) {
      llvm::Value *stride = getStride(IGF, T);
      llvm::Value *byteCount = IGF.Builder.CreateNUWMul(stride, count);
      IGF.Builder.CreateMemCpy(dest.getAddress(),
                               dest.getAlignment().getValue(),
                               src.getAddress(),
                               src.getAlignment().getValue(), byteCount);
      return;
   }

   emitAssignArrayWithTakeCall(IGF, T, dest, src, count);
}

void TypeInfo::collectMetadataForOutlining(OutliningMetadataCollector &c,
                                           PILType T) const {
   auto canType = T.getAstType();
   assert(!canType->is<ArchetypeType>() && "Did not expect an ArchetypeType");
   (void)canType;
}
